package com.lixuwei.study.htmlutil;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.TypeReference;
import com.gargoylesoftware.htmlunit.*;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlSpan;
import com.gargoylesoftware.htmlunit.util.Cookie;
import com.gargoylesoftware.htmlunit.util.UrlUtils;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;

import javax.script.Invocable;
import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.HTTP_HEADER_UPGRADE_INSECURE_REQUEST;

public class Hello {

    private static String shopListUrl = "https://shop{shopNum}.m.taobao.com/?shop_id={shopNum}#list";
    private static String homeUrl = "https://shop{shopNum}.m.taobao.com/";

    private static Pattern pattern = Pattern.compile(".*?id=(\\d+\\d$)");
    private static Pattern jsonpPattern = Pattern.compile("mtopjsonp1\\((.*)\\)");

    public static void main(String[] args) throws Exception {
//        parseShop("https://shop62745329.m.taobao.com/?shop_id=62745329&user_id=165257719&spm=a2141.7631565.shop_head_17386384476.0#list");
//        parseHome("62745329");

//        ScriptEngine engine = new ScriptEngineManager().getEngineByName("nashorn");
//        URL url = Hello.class.getClassLoader().getResource("sign.js");
//        engine.eval(new FileReader(new File(url.getFile())));
//        Invocable invocable = (Invocable) engine;
//        Object result = invocable.invokeFunction("h", "abc");
//        System.out.println(result.toString());
//
//        String encodeStr= DigestUtils.md5Hex("abc");
//        System.out.println(encodeStr);


//        System.out.println((int) (Math.random() * 1000 + 2000));

//        List<Integer> pageNoSet = new ArrayList<>();
//        for (int i = 0; i < 50; i++) {
//            pageNoSet.add(i);
//        }
//        Collections.shuffle(pageNoSet);
//        pageNoSet.forEach(System.out::println);

//        int lucky = (int) (Math.random() * 10) + 10;
//        System.out.println(lucky);


//        Random random = new Random();
//        for (int i = 0; i < 10; i++) {
//            int a = random.nextInt(9) + 10;
//            System.out.println(a);
//        }
//        String url = "https://h5api.m.taobao.com/h5/com.taobao.search.api.getshopitemlist/2.0/?jsv=2.4.2&appKey=12574478&t=1536207026780&sign=886eb850e7a23faaa1cadf6b1500b757&api=com.taobao.search.api.getShopItemList&v=2.0&type=jsonp&dataType=jsonp&callback=mtopjsonp1&data=";
//        String data = "{\"shopId\":\"62745329\",\"currentPage\":1,\"pageSize\":\"30\",\"sort\":\"oldstarts\",\"q\":\"\"}";
//        String encode = URLEncoder.encode(data, "UTF-8");
//        System.out.println(url+encode);




    }


    private static void parseHome(String shopNum) throws Exception {
        String url = getShopHomeUrl(shopNum);
        parseShopHome(url, shopNum);
    }


    private static void parseShopHome(String url, String shopNum) throws IOException, ScriptException, NoSuchMethodException, InterruptedException {
        System.out.println(url);
        try (WebClient webClient = getWebClient()) {
            HtmlPage page = webClient.getPage(url);
            CookieManager cookieManager = page.getWebClient().getCookieManager();
            Set<Cookie> cookies = cookieManager.getCookies();
            cookies.forEach(System.out::println);
            Cookie m_h5_tk = cookieManager.getCookie("_m_h5_tk");
            //得到token
            String[] vals = m_h5_tk.getValue().split("_");
            String token = vals[0];
            String tokenExpireTime = vals[1];
            System.out.println(token);
            System.out.println(tokenExpireTime);
            Thread.sleep(1000);
            String itmeListJsonp = getItmeListJsonp(shopNum, token, webClient, 1);
            System.out.println(itmeListJsonp);
            String resultStr = parseJsonp(itmeListJsonp);
            HashMap<String, Object> resultMap = JSON.parseObject(resultStr, new TypeReference<HashMap<String, Object>>() {});
            Map<String, Object> data = (Map<String, Object>) resultMap.get("data");
            Object totalResultsStr = data.get("totalResults");
            Integer totalResults = Integer.parseInt(totalResultsStr.toString());
            System.out.println("第1页");
            parseProductIds(data);
            Integer totalPage = (totalResults % 30) == 0 ? (totalResults / 30) : ((totalResults / 30) + 1);
            if (totalPage > 1) {
                for (Integer pageNo = 2; pageNo <= totalPage; pageNo++) {
                    Thread.sleep(1500);
                    System.out.println("第"+pageNo+"页");
                    itmeListJsonp = getItmeListJsonp(shopNum, token, webClient, pageNo);
                    System.out.println(itmeListJsonp);
                    resultStr = parseJsonp(itmeListJsonp);
                    resultMap = JSON.parseObject(resultStr, new TypeReference<HashMap<String, Object>>() {
                    });
                    data = (Map<String, Object>) resultMap.get("data");
                    parseProductIds(data);
                }
            }
        }
    }

    private static void parseProductIds(Map<String, Object> data) {
        List<Map<String, Object>> itemsArray = (List<Map<String, Object>>) data.get("itemsArray");
        for (Map<String, Object> item : itemsArray) {
            String productId = item.get("auctionId").toString();
            System.out.print(productId + ", ");
        }
        System.out.println("");
    }

    private static String parseJsonp(String itmeListJsonp) {
        Matcher matcher = jsonpPattern.matcher(itmeListJsonp);
        if (matcher.find()) {
            return matcher.group(1);
        }
        return "";
    }

    private static String getItmeListJsonp(String shopNum, String token, WebClient webClient, int pageNo) throws IOException {
        String appkey = "12574478";
        long currentTimeMillis = System.currentTimeMillis();
        String api = "com.taobao.search.api.getShopItemList";

        Map<String, Object> queryData = new LinkedHashMap<>();
        queryData.put("shopId", shopNum);
        queryData.put("currentPage", pageNo);
        queryData.put("pageSize", "30");
        queryData.put("sort", "oldstarts");
        queryData.put("q", "");
        String queryDataJson = JSON.toJSONString(queryData);
        StringBuffer signSb = new StringBuffer("");
        signSb.append(token).append("&").append(currentTimeMillis)
                .append("&").append(appkey).append("&").append(queryDataJson);

        String sign = DigestUtils.md5Hex(signSb.toString());

        String itemListUrlStr = "https://h5api.m.taobao.com/h5/com.taobao.search.api.getshopitemlist/2.0/";
        StringBuilder itemListSb = new StringBuilder(itemListUrlStr);
        itemListSb.append("?jsv=2.4.2&appKey=").append(appkey)
                .append("&t=").append(currentTimeMillis).append("&sign=").append(sign)
                .append("&api=").append(api).append("&v=2.0&type=jsonp&dataType=jsonp&callback=mtopjsonp1&data=")
                .append(queryDataJson);
        System.out.println(itemListSb.toString());
//        URL itemListUrl = UrlUtils.toUrlUnsafe(itemListSb.toString());
//        WebRequest webRequest = new WebRequest(itemListUrl);
//        //TODO 为webRequest添加header 比如移动端的头
//        addDefaultHeaders(webClient, webRequest);
//
//        WebResponse response = webClient.getWebConnection().getResponse(webRequest);
//        return response.getContentAsString();

        Page jsonpPage = webClient.getPage(itemListSb.toString());
        System.out.println(jsonpPage.getWebResponse().getContentAsString());
        return "";
    }

    private static void addDefaultHeaders(WebClient webClient, WebRequest wrs) {
        BrowserVersion browserVersion = webClient.getBrowserVersion();
        // Add standard HtmlUnit headers.
        if (!wrs.isAdditionalHeader(HttpHeader.ACCEPT_LANGUAGE)) {
            wrs.setAdditionalHeader(HttpHeader.ACCEPT_LANGUAGE, browserVersion.getBrowserLanguage());
        }
        if (browserVersion.hasFeature(HTTP_HEADER_UPGRADE_INSECURE_REQUEST)
                && !wrs.isAdditionalHeader(HttpHeader.UPGRADE_INSECURE_REQUESTS)) {
            wrs.setAdditionalHeader(HttpHeader.UPGRADE_INSECURE_REQUESTS, "1");
        }

        // TODO 添加自定义的http头
//        wrs.setAdditionalHeader();
//        wrs.setAdditionalHeader("user-agent", "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Mobile Safari/537.36");
//      wrs.setAdditionalHeader("accept-language", "zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7");
    }

    private static String getShopHomeUrl(String shopNum) {
        return homeUrl.replace("{shopNum}", shopNum);
    }

    private static void parseList(String shopNum) throws Exception {
        String url = getShopListUrl(shopNum);
        parseShop(url);
    }

    private static WebClient getWebClient() {
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        webClient.getOptions().setJavaScriptEnabled(true);
        webClient.getOptions().setCssEnabled(false);
        webClient.getOptions().setActiveXNative(false);
        webClient.getOptions().setCssEnabled(false);
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
        webClient.getOptions().setTimeout(5000);
        webClient.getOptions().setDownloadImages(false);
        webClient.waitForBackgroundJavaScript(5000);
//        ProxyConfig proxyConfig = new ProxyConfig();
//        proxyConfig.setProxyHost("101.68.78.190");
//        proxyConfig.setProxyPort(808);
//        webClient.getOptions().setProxyConfig(proxyConfig);
        webClient.setAjaxController(new NicelyResynchronizingAjaxController());
        webClient.addRequestHeader("accept-language", "zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7");

        return webClient;
    }

    //HtmlPage page = webClient.getPage("https://shop486380875.m.taobao.com/?shop_id=486380875&user_id=3232205451&spm=a2141.7631565.shop_head_18337304155.0#list");
    //HtmlPage page = webClient.getPage("https://shop35061001.m.taobao.com/?shop_id=35061001&user_id=66695381#list");
    public static List<String> parseShop(String url) throws Exception {
        try (WebClient webClient = getWebClient()) {
            HtmlPage page = webClient.getPage(url);

            Set<String> productIds = new HashSet<>();
            if (page != null) {
                parseItem(page, productIds);
                int pageSize = parseItemPageSize(page);
                if (pageSize > 0) {
                    //从第二页开始
                    for (int i = 2; i <= pageSize; i++) {
                        if (page != null) {
                            //睡3秒
                            Thread.sleep(3000);
                            System.out.println("第" + i + "页 ");
                            page = getNextPage(page);
                        }
                        if (page != null) {
                            parseItem(page, productIds);
                        }
                    }
                }
            }
            return new ArrayList<>(productIds);
        }
    }

    private static HtmlPage getNextPage(HtmlPage page) throws IOException {
        //寻找页码 进行点击事件
        DomElement pageNavBar = page.getElementById("gl-pagenav");
        DomElement nextPageBtn = pageNavBar.getFirstElementChild().getLastElementChild();
        String attribute = nextPageBtn.getAttribute("class");
        if (attribute.contains("c-btn-off")) {
            return null;
        }
        return nextPageBtn.click();
    }

    private static int parseItemPageSize(HtmlPage page) {
        List<DomElement> options = page.getByXPath("//*[@id='gl-pagenav']/section/div/span/select/option");
        if (options != null) {
            return options.size();
        }
        return 0;
    }

    private static void paseNexPage(HtmlPage page) throws IOException {
        //寻找页码 进行点击事件
        DomElement pageNavBar = page.getElementById("gl-pagenav");
        DomElement nextPageBtn = pageNavBar.getFirstElementChild().getLastElementChild();
        String attribute = nextPageBtn.getAttribute("class");
        if (attribute.contains("c-btn-off")) {
            return;
        }
        HtmlPage clickPage = nextPageBtn.click();
        parseItem(clickPage, new HashSet<>());
        paseNexPage(clickPage);
    }

    private static void parseItem(HtmlPage page, Set<String> productIds) {
        List<DomElement> liElements = page.getByXPath("//*[@id='js-goods-list-items']/li");
        liElements.forEach(domElement -> {
            String href = domElement.getFirstElementChild().getAttribute("href");
            if (StringUtils.isNotEmpty(href)) {
                //https://h5.m.taobao.com/awp/core/detail.htm?id=573360510637
                String id = parseId(href);
                if (StringUtils.isNotEmpty(id)) {
                    productIds.add(id);
                    System.out.print(id + " ,");
                }
            }
        });
        System.out.println("");
    }

    private static String parseId(String href) {
        Matcher matcher = pattern.matcher(href);
        if (matcher.find()) {
            return matcher.group(1);
        }
        return "";
    }

    public static String getShopListUrl(String shopNum) {
        return shopListUrl.replace("{shopNum}", shopNum);
    }
}